{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五步：调整树的参数：subsample 和 colsample_bytree\n",
    "(粗调，参数的步长为0.1；下一步是在粗调最佳参数周围，将步长降为0.05，进行精细调整)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = np.array(train.drop([\"interest_level\"], axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二轮参数调整得到的n_estimators最优值（316），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n",
       " 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zhangwt/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:762: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58718, std: 0.00432, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58535, std: 0.00417, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58400, std: 0.00339, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58341, std: 0.00427, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58316, std: 0.00362, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58159, std: 0.00375, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58737, std: 0.00324, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58440, std: 0.00401, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58382, std: 0.00366, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58271, std: 0.00236, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58253, std: 0.00397, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58181, std: 0.00428, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58782, std: 0.00517, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58560, std: 0.00442, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58503, std: 0.00403, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58312, std: 0.00302, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58273, std: 0.00373, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58269, std: 0.00368, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58758, std: 0.00353, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58532, std: 0.00333, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58280, std: 0.00412, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58351, std: 0.00364, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58275, std: 0.00391, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58172, std: 0.00340, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       " -0.5815879461119965)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=316,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
